package cn.edu360.day08;

import cn.edu360.day08.bean.Job;
import cn.edu360.day08.bean.Page;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * 爬取岗位数据
 */
public class TestMain {

    public static void main(String[] args) {
        String urlHrb = "https://search.51job.com/list/220200,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
        String url = "https://search.51job.com/list/010000,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,1.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=4&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
        String endUrl = "https://search.51job.com/list/010000,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,145.html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=4&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=";
        Document document = getDocumentByUrl(urlHrb);
        //System.out.println(document);
        Page page = new Page();
        page.setDocument(document);
        //System.out.println(page);
        int count = 0;
        while(true){
            System.out.println("第"+(++count)+"页");
            //获取岗位数据
            List<Job> jobByPage = getJobByPage(page);
            //岗位数据需要处理的，保存到文件，或者到数据库
            for (Job job : jobByPage) {
                System.out.println(job);
            }
            //获取下一个url和是否有下一页    值存储在page里面
            getNextUrlByPage(page);
            if(page.isHasNextPage()){//有下一页
                Document documentByUrl = getDocumentByUrl(page.getUrl());
                page.setDocument(documentByUrl);
            }else{//没有下一页，结束程序
                break;
            }
            //休眠一秒
            try{
                Thread.sleep(1000);
            }catch (Exception e){}

        }

    }

    /**
     * 获取下一页url和判断是否有下一页
     * @param page
     */
    public static void getNextUrlByPage(Page page){
        Document document = page.getDocument();
        Elements list = document.select(".bk");
        //获取索引值为1的数据
        Element element = list.get(1);
        String url = element.select("a").attr("href");
        if(url!=null&&!"".equals(url)){//有数据的时候
            page.setUrl(url);
            page.setHasNextPage(true);
        }else{
            page.setHasNextPage(false);
        }
        //System.out.println(url);
    }

    /**
     * 通过document获取岗位数据
     * @param page
     * @return
     */
    public static List<Job> getJobByPage(Page page){
        //存储job数据
        ArrayList<Job> list = new ArrayList<>();
        //获取document
        Document document = page.getDocument();
        Elements select = document.select("#resultList .el");
        //删除第一个数据（标题）
        select.remove(0);
        for (Element element : select) {
            Elements jobElement = element.select(".t1 a");
            String jobName = jobElement.attr("title");
            String comName = element.select(".t2 a").attr("title");
            String addr = element.select(".t3").text();
            String price = element.select(".t4").text();
            String date = element.select(".t5").text();
            Job job = new Job();
            job.set(jobName,comName,addr,price,date);
            list.add(job);
            //System.out.println(addr);
        }
        return  list;
    }

    /**
     * 通过url获取到document
     * @param url
     * @return
     */
    public static Document getDocumentByUrl(String url){
        try{
            Document document = Jsoup.parse(new URL(url), 6000);
            return document;
        }catch (Exception e){
            e.printStackTrace();
        }
        return  null;
    }
}
